# -*- coding: utf-8 -*-
import re
from time import sleep, time
import pandas as pd
import os
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from selenium.webdriver.support.select import Select
from config import url,university,username,password,keyword,countrys
import requests
from lxml import etree
from retrying import retry

wait_time = 60
wait_time0 = 3
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
datas_file = os.path.join(BASE_DIR, "datas.xlsx")
writer = pd.ExcelWriter(datas_file)
results_list = []
headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36",
    "Accept-Encoding": "gzip, deflate, br",
    "Host": "https://apps.webofknowledge.com/summary.do?product=UA&doc=1&qid=1&SID=5DESDnu1UoupKtIMjjd&search_mode=AdvancedSearch&update_back2search_link_param=yes",
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9"
}


def save_data(writer, datas):
    if os.path.exists(datas_file):
        try:
            df = pd.read_excel(datas_file)
            df = df.append(datas)
        except:
            df = pd.DataFrame(datas)
    else:
        df = pd.DataFrame(datas)
    df.to_excel(excel_writer=writer, columns=[i for i in datas[0].keys()], index=False, encoding="utf-8")


def run():
    options = webdriver.ChromeOptions()
    options.add_argument('--no-sandbox')
    dr = webdriver.Chrome(options=options)
    # dr = webdriver.Chrome()
    dr.maximize_window()
    dr.get(url)
    # 选择机构登录
    opt = WebDriverWait(dr, wait_time).until(
        lambda driver: dr.find_element_by_xpath('//select[@id="shibSelect"]'))
    Select(opt).select_by_visible_text("CHINA CERNET Federation")
    WebDriverWait(dr, wait_time).until(
        lambda driver: dr.find_element_by_xpath('//button[@id="shibSubmit"]')).click()
    # 输入学校名称
    WebDriverWait(dr, wait_time).until(
        lambda driver: dr.find_element_by_xpath('//input[@id="idpSelectInput"]')).send_keys(university)
    WebDriverWait(dr, wait_time).until(
        lambda driver: dr.find_element_by_xpath('//input[@id="idpSelectSelectButton"]')).click()
    # 登录
    WebDriverWait(dr, wait_time).until(
        lambda driver: dr.find_element_by_xpath('//input[@id="username"]')).send_keys(username)
    WebDriverWait(dr, wait_time).until(
        lambda driver: dr.find_element_by_xpath('//input[@id="password"]')).send_keys(password)
    WebDriverWait(dr, wait_time).until(
        lambda driver: dr.find_element_by_xpath('//button[@class="form-element form-button"]')).click()
    sleep(2)
    WebDriverWait(dr, wait_time).until(
            EC.element_to_be_clickable((By.XPATH, '//input[@value="同意"]'))).click()
    sleep(3)
    # 高级检索
    WebDriverWait(dr, wait_time).until(
            EC.element_to_be_clickable((By.XPATH, '//div[@class="searchtype-sub-nav"]//a[text()="高级检索"]'))).click()
    # 输入关键词
    WebDriverWait(dr, wait_time).until(
        lambda driver: dr.find_element_by_xpath('//div[@class="AdvSearchBox"]/textarea[@id="value(input1)"]')).send_keys(keyword)
    WebDriverWait(dr, wait_time).until(
        EC.element_to_be_clickable((By.XPATH, '//div[@class="AdvSearchBox"]//button[@id="search-button"]'))).click()
    # 点击检索结果
    WebDriverWait(dr, wait_time).until(
        EC.element_to_be_clickable((By.XPATH, '//div[@class="block-history"]//div[@id="set_1_div"]/a[@id="hitCount"]'))).click()
    # 选择出版年
    WebDriverWait(dr, wait_time).until(
        EC.element_to_be_clickable((By.XPATH, '//a[@id="PublicationYear"]'))).click()
    WebDriverWait(dr, wait_time).until(
        EC.element_to_be_clickable((By.XPATH, '//input[@value="PublicationYear_2014"]'))).click()
    WebDriverWait(dr, wait_time).until(
        EC.element_to_be_clickable((By.XPATH, '//td[@class="ra_button_row"]//button[@title="精炼"]'))).click()
    # 选择国家
    WebDriverWait(dr, wait_time).until(
        EC.element_to_be_clickable((By.XPATH, '//div[@class="moreRAOptsLink"]/a'))).click()
    WebDriverWait(dr, wait_time).until(
        EC.element_to_be_clickable((By.XPATH, '//div[@id="CountryTerritory"]/h4/a'))).click()
    WebDriverWait(dr, wait_time).until(
        EC.element_to_be_clickable((By.XPATH, '//div[@id="CountryTerritory"]//div[@id="CountryTerritory_tr"]//a[@title="更多精炼检索结果"]'))).click()
    for country in countrys:
        country = country.upper()
        WebDriverWait(dr, wait_time).until(
            EC.element_to_be_clickable((By.XPATH, '//input[@value="CountryTerritory_{}"]'.format(country)))).click()
    WebDriverWait(dr, wait_time).until(
        EC.element_to_be_clickable((By.XPATH, '//td[@class="ra_button_row"]//button[@title="精炼"]'))).click()
    sleep(5)
    # 提取数据
    total_num = WebDriverWait(dr, wait_time).until(
        lambda driver: dr.find_element_by_xpath('//*[@id="hitCount.top"]')).text
    total_num = int(total_num.replace(",", ""))
    detail_herf = WebDriverWait(dr, wait_time).until(
        lambda driver: dr.find_element_by_xpath('//div[@id="RECORD_1"]//a[@class="smallV110 snowplow-full-record"]')).get_attribute("href")
    for i in range(1, total_num + 1):
        results_dict = {}
        # try:
        #     WebDriverWait(dr, wait_time).until(
        #         lambda driver: dr.find_element_by_xpath(
        #             '//div[@id="RECORD_{}"]//a[@class="smallV110 snowplow-full-record"]'.format(i))).click()
        #     sleep(3)
        # except:
        #     print("第{}项点击进入详情页失败".format(i))
        #     continue
        page = i // 10 + 1
        detail_url = detail_herf.replace("page=1&doc=1", "page={}&doc={}".format(page, i))
        dr.get(detail_url)
        sleep(3)
        try:
            # title = WebDriverWait(dr, wait_time).until(
            #     lambda driver: dr.find_element_by_xpath(
            #         '//div[@class="title"]/value')).text
            title = WebDriverWait(dr, wait_time).until(
                lambda driver: dr.find_element_by_xpath(
                    '//div[@class="title"]')).text
        except:
            title = ""
        try:
            label = WebDriverWait(dr, wait_time0).until(
                lambda driver: dr.find_element_by_xpath(
                    '//span[text()="作者:"]/..'))
            labels = label.text.split("; ")
            if len(labels) > 3:
                label_list = labels[:3]
            label_str = ""
            for lab in label_list:
                label_str += re.search(".*(\(.*\)).*", lab).group(1)
        except:
            label_str = ""
        try:
            year = WebDriverWait(dr, wait_time0).until(
                lambda driver: dr.find_element_by_xpath(
                    "//span[contains(text(),'出版年')]/../value")).text
        except:
            year = ""
        try:
            type = WebDriverWait(dr, wait_time0).until(
                lambda driver: dr.find_element_by_xpath(
                    '//*[text()="文献类型:"]/..'))
            type = type.text.split(":")[1]
        except:
            type = ""
        try:
            fr_label = WebDriverWait(dr, wait_time0).until(
                lambda driver: dr.find_element_by_xpath('//span[contains(text(),"通讯作者地址:")]/..')).text
            fr_label = fr_label.split(":")[1].split(" (")[0].strip()
            for ll in label_list:
                if fr_label in ll:
                    fr_label = re.search(".*(\(.*\)).*", ll).group(1)
                    break
        except:
            fr_label = ""
        try:
            fr_address = WebDriverWait(dr, wait_time0).until(
                lambda driver: dr.find_element_by_xpath(
                    '//*[@id="records_form"]//table[@class="FR_table_noborders"][1]/tbody/tr/td[2]')).text
        except:
            fr_address = ""
        try:
            fr_address_1 = WebDriverWait(dr, wait_time0).until(
                lambda driver: dr.find_element_by_xpath('//*[@id="records_form"]//table[@class="FR_table_noborders"][2]/tbody/tr[1]//a')).text
        except:
            fr_address_1 = ""
        try:
            large_number = WebDriverWait(dr, wait_time0).until(
                lambda driver: dr.find_element_by_xpath(
                    '//div[@class="flex-row flex-justify-start flex-align-start box-div"]//span[@class="large-number"]')).text
        except:
            large_number = ""
        try:
            publisher = WebDriverWait(dr, wait_time0).until(
                lambda driver: dr.find_element_by_xpath(
                    '//div[text()="出版商"]/../p/value')).text
        except:
            publisher = ""
        try:
            fund_links = WebDriverWait(dr, wait_time0).until(
                lambda driver: dr.find_elements_by_xpath(
                    '//th[text()="基金资助机构"]/../../tr[@class="fr_data_row"]'))
            fund_link = ""
            for link in fund_links[:3]:
                fund_link += WebDriverWait(link, wait_time0).until(
                    lambda driver: link.find_element_by_xpath(
                        './td')).text + " "
        except:
            fund_link = ""
        # 点击更多数据字段(提取PubMed ID)
        try:
            if i == 1:
                WebDriverWait(dr, wait_time).until(
                    EC.element_to_be_clickable((By.XPATH, '//a[@id="hidden_section_label"]'))).click()
            sleep(2)
            PubMed_ID = WebDriverWait(dr, wait_time0).until(
                    lambda driver: dr.find_element_by_xpath(
                        '//*[text()="PubMed ID:"]/../value')).text
        except:
            PubMed_ID = ""
        # 保存数据
        results_dict["标题"] = title
        results_dict["作者姓名"] = label_str
        results_dict["出版年"] = year
        results_dict["文献类型"] = type
        results_dict["通讯作者"] = fr_label
        results_dict["通讯作者地址"] = fr_address
        results_dict["第一作者地址"] = fr_address_1
        results_dict["核心期刊被引次数"] = large_number
        results_dict["PubMed_ID"] = PubMed_ID
        results_dict["基金资助机构"] = fund_link
        results_dict["出版商"] = publisher
        print(i, results_dict)
        results_list.append(results_dict)
        # all_vals = results_dict.values()
        # all_vals_list = []
        # for av in all_vals:
        #     if av:
        #         all_vals_list.append(av)
        # if all_vals_list:
        #     dr.back()
        # if i % 10 == 0:
        #     try:
        #         page_input = WebDriverWait(dr, wait_time).until(
        #             lambda driver: dr.find_element_by_xpath(
        #                 '//input[@class="goToPageNumber-input"]'))
        #         page_input.clear()
        #         page_input.send_keys(page)
        #         page_input.send_keys(Keys.ENTER)
        #         sleep(5)
        #     except:
        #         print("第{}页跳转失败,请求重新运行脚本".format(i//10 + 1))
        #         break


    dr.quit()


if __name__ == '__main__':
    start = time()
    try:
        run()
    except:
        pass
    save_data(writer, results_list)
    writer.save()
    end = time()
    print("delta耗时:{}S".format(end-start))